from bs4 import BeautifulSoup
import urllib3
import time
from jkdb.jkdb import jkdb
import json

## 从某网站上提取出一些大学信息，仅学习之用，勿损坏此网站服务，勿做违法行为

def save_tomongo(data):
    db = jkdb("mongo", "127.0.0.1", 27017)
    db.auth('', '', 'proj')
    # print(type(data))
    # print(json.dumps(data))
    db.insert('proj', 'daxue', data)


def save_local(localfile, data):
    f = open(localfile, 'w')
    f.write(data)
    f.close()


def read_local(file):
    f = open(file, 'r')
    data = f.read()
    f.close()
    return data


def get_daxue(index):
    url = ""
    if index >= 1:
        url += "/p" + str(index)
    print("url: ", url)
    http = urllib3.PoolManager()
    r = http.request('GET', url)
    save_local('./data/.data' + str(index), r.data.decode('gb2312'))


def get_daxue_all():
    for i in range(1, 108):
        get_daxue(i)
        time.sleep(0.2)


resultlistall = []
insert_count_all = 0
insert_skip_all = 0

def parse_data(file_index):
    print("parse data ./data/.data" + str(file_index))
    data = read_local('./data/.data' + str(file_index))
    soup = BeautifulSoup(data, "html.parser")
    list = soup.find('div', class_="scores_List")
    print("len ", len(list))
    resultlist = []
    for x in list:
        result = {}
        strong = x.find('strong')
        if not isinstance(strong, int) and strong is not None and strong.has_attr('title'):
            result["name"] = strong['title']
        subul = x.find('ul')
        if subul != -1 and subul is not None:
            index = 0
            for subli in subul.find_all('li'):
                ss = subli.text.split('：')
                if len(ss) > 1:
                    if index == 0:
                        result["suozaidi"] = ss[1]
                    elif index == 2:
                        result["leixing"] = ss[1]
                    elif index == 3:
                        result["lishu"] = ss[1]
                    elif index == 4:
                        result["xingzhi"] = ss[1]
                    elif index == 5:
                        result["wangzhi"] = ss[1]
                    elif index == 1:
                        result["211"] = False
                        result["985"] = False
                        if ss[1].find("211") != -1:
                            result["211"] = True
                        if ss[1].find("985") != -1:
                            result["985"] = True
                index += 1

        resultlist.append(result)
        resultlistall.append(result)

    print("done for size ", len(resultlist))

    insert_count = 0
    skip_count = 0
    for rl in resultlist:
        if rl != {}:
            # print(rl)
            caninsert = True
            for j in range(0, len(resultlistall) - len(resultlist)):
                if resultlistall[j] != {} and rl['name'] == resultlistall[j]['name']:
                    caninsert = False
                    skip_count += 1
            if caninsert:
                insert_count += 1
                save_tomongo(rl)

    print("insert count ", insert_count, ", skip count ", skip_count, ", result all ", len(resultlistall))


def parse_data_all():
    for i in range(0, 108):
        parse_data(i)


if __name__ == '__main__':
    # get_daxue_all()
    parse_data_all()
    # get_daxue(100)
